<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2018 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Field Level Lineage</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="Metadata" href="index.html" />
    <link rel="next" title="Audit Logging" href="audit-logging.html" />
    <link rel="prev" title="Discovery and Lineage" href="discovery-lineage.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: developer-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../getting-started/index.html"> 入门指南</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/sandbox/index.html">CDAP Sandbox</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html">二进制 Zip 文件</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html#cdap-sandbox">启动和停止 CDAP Sandbox</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/virtual-machine.html">虚拟机镜像</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/docker.html">Docker 镜像</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/quick-start.html">快速入门</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/dev-env.html">搭建开发环境</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/start-stop-cdap.html">启动和停止 CDAP</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/building-apps.html">构建并运行应用</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../overview/index.html"> 概述</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../overview/anatomy.html"> 大数据应用剖析</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/modes.html"> 模式和组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/abstractions.html"> 核心概念</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/interfaces.html"> 编程接口</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../building-blocks/index.html"> 抽象概念</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/core.html"> Core Abstractions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/applications.html"> Applications</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/datasets/index.html"> Datasets</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/overview.html"> Overview</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/table.html"> Table API</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/fileset.html"> FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/partitioned-fileset.html"> Partitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/time-partitioned-fileset.html"> TimePartitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/system-custom.html"> System and Custom Datasets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/permissions.html"> Dataset Permissions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/cube.html"> Cube Dataset</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/mapreduce-programs.html"> MapReduce Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/plugins.html"> Plugins</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/schedules.html"> Schedules</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/secure-keys.html"> Secure Keys</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/services.html"> Services</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/spark-programs.html"> Spark Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/workers.html"> Workers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/workflows.html"> Workflows</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/artifacts.html"> Artifacts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/program-lifecycle.html"> Program Lifecycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/namespaces.html"> Namespaces</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/transaction-system.html"> Transaction System</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/transactional-messaging-system.html"> Transactional Messaging System</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> 元数据</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="system-metadata.html"> System Metadata</a></li>
<li class="toctree-l2"><a class="reference internal" href="discovery-lineage.html"> Discovery and Lineage</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#"> Field Level Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="audit-logging.html"> Audit Logging</a></li>
<li class="toctree-l2"><a class="reference internal" href="metadata-ui.html"> CDAP Metadata UI</a></li>
<li class="toctree-l2"><a class="reference internal" href="programmatic-metadata.html"> Accessing metadata programmatically</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../pipelines/index.html"> 数据流管道</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/concepts-design.html"> Concepts and Design</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/getting-started.html"> Getting Started</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/studio.html"> CDAP Studio</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/creating-pipelines.html"> Creating Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/running-pipelines.html"> Running Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/plugin-management.html"> Plugin Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/plugins/index.html"> Plugin Reference</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/actions/index.html"> Action Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/sources/index.html"> Source Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/transforms/index.html"> Transform Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/analytics/index.html"> Analytic Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/sinks/index.html"> Sink Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/shared-plugins/index.html"> Shared Plugins</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../pipelines/plugins/shared-plugins/core.html">CoreValidator</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/post-run-plugins/index.html"> Post-run Plugins</a><ul class="simple">
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/developing-pipelines.html"> Developing Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/developing-plugins/index.html"> Developing Plugins</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/plugin-basics.html">Plugin Basics</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/creating-a-plugin.html">Creating a Plugin</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/presentation-plugins.html">Plugin Presentation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/testing-plugins.html">Testing Plugins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/packaging-plugins.html">Packaging Plugins</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/how-cdap-pipelines-work.html"> How CDAP Pipelines Work</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../cloud-runtimes/index.html"> 云平台运行</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/concepts/index.html"> Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/provisioners/index.html"> Provisioners</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/gcp-dataproc.html">Google Dataproc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/aws-emr.html">Amazon Elastic MapReduce</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/remote-hadoop.html">Remote Hadoop</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/profiles/index.html"> Profiles</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/creating-profiles.html">Creating Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/assigning-profiles.html">Assigning Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/admin-controls.html">Admin Controls</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/example/index.html"> Example</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../security/index.html"> 安全</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../security/client-authentication.html">Client Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-java.html">CDAP Authentication Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-python.html">CDAP Authentication Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/custom-authentication.html">Custom Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/authorization-extensions.html">Authorization Extensions</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../testing/index.html"> 测试和调试</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../testing/testing.html"> Testing a CDAP Application</a></li>
<li class="toctree-l2"><a class="reference internal" href="../testing/debugging.html"> Debugging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../testing/troubleshooting.html"> Troubleshooting</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../ingesting-tools/index.html"> 数据融合</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-java.html">CDAP Stream Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-python.html">CDAP Stream Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-ruby.html">CDAP Stream Client for Ruby</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-flume.html">CDAP Flume</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../data-exploration/index.html"> 数据探索</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/filesets.html"> Fileset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/tables.html"> Table Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/object-mapped-tables.html"> ObjectMappedTable Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/custom-datasets.html"> Custom Dataset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/hive-execution-engines.html"> Hive Execution Engines</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/index.html"> 高级主题</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../advanced/application-logback.html"> Application Logback</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/best-practices.html"> Best Practices</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/class-loading.html"> Class Loading</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/configuring-resources.html"> Configuring Program Resources</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/program-retry-policies.html"> Program Retry Policies</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="field-level-lineage">
<h1>Field Level Lineage<a class="headerlink" href="#field-level-lineage" title="Permalink to this headline">🔗</a></h1>
<div class="section" id="introduction">
<span id="metadata-fieldlevellineage"></span><h2>Introduction<a class="headerlink" href="#introduction" title="Permalink to this headline">🔗</a></h2>
<p>CDAP provides a way to retrieve the lineage for dataset entities. A dataset entity can have associated schema.
The schema defines different fields in the dataset along with their data type information.
<strong>Field Level Lineage</strong> allows a user to get more granular lineage view of a dataset. A field lineage for a
given dataset shows for the specified time range all the fields that were computed for a dataset and the fields
from source datasets that participated in computation of those fields. Field lineage also shows the detail
operations that caused the transformation from fields of a source dataset to the field of a given dataset.</p>
</div>
<div class="section" id="example-use-case">
<h2>Example Use Case<a class="headerlink" href="#example-use-case" title="Permalink to this headline">🔗</a></h2>
<p>Data analytics group in an Airline company wants to perform analytical queries on the passenger
information stored on SFTP servers. To run the workload, passenger information is imported to Hadoop.
Because the data can come from a variety of sources, the data is normalized into standard format while
importing. PII information is also obfuscated.</p>
<p>By looking at the dataset in Hadoop, a data officer or Business Analyst wants to understand the meaning of the field
“fullName” by reading how it was produced. For example, a data officer might want to know that the “fullName”
field was created by concatenating the “firstName” and “lastName” fields, both of which were extracted as positional
fields from a CSV record in the source named “passengerList”. Additionally, typically in a triage or debugging
scenario, an operation team or developer wants to identify how a field was computed when fields show up with wrong
values.</p>
</div>
<div class="section" id="concepts-and-terminology">
<h2>Concepts and Terminology<a class="headerlink" href="#concepts-and-terminology" title="Permalink to this headline">🔗</a></h2>
<ul class="simple">
<li><strong>Field</strong> : Field identifies column in a dataset. Field has a name and data type.</li>
<li><strong>EndPoint</strong> : EndPoint defines the source or destination of the data along with its namespace from where the fields are read or written to.</li>
<li><strong>Field Operation</strong> : Operation defines a single computation on a field. It has a name and description.</li>
<li><strong>Read Operation</strong> : Type of operation that reads from the source EndPoint and creates collection of fields.</li>
<li><strong>Transform Operation</strong> : Type of operation that transforms collection of input fields to collection of output fields.</li>
<li><strong>Write Operation</strong> : Type of operation that writes the collection of fields to the destination EndPoint.</li>
<li><strong>Origin</strong> : Origin of the field is the name of the operation that outputted the field. The &lt;origin, fieldName&gt; pair is used to uniquely identify the field because the field can appear in the outputs of multiple operations.</li>
</ul>
</div>
<div class="section" id="field-lineage-for-cdap-programs">
<h2>Field Lineage for CDAP Programs<a class="headerlink" href="#field-lineage-for-cdap-programs" title="Permalink to this headline">🔗</a></h2>
<p>Field Lineage recording is supported from MapReduce and Spark programs.</p>
<p><strong>Note:</strong> All the operations that CDAP programs record must have unique names.</p>
<p>Consider a simple MapReduce program that reads the passenger information stored in CSV files and normalizes
the data into standard format before storing it in the CDAP dataset. The program assumes that the fields in the
CSV files are as follows: id, firstName, lastName, address. The program then concatenate firstName and lastName to
create fullName. The address field is normalized into standard format. In this case, the program can record the
following field operations:</p>
<p>The <code class="docutils literal notranslate"><span class="pre">ReadOperation</span></code> to represent read from <code class="docutils literal notranslate"><span class="pre">passengerList</span></code> file to create fields id, firstName, lastName,
and address:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">Operation</span> <span class="n">read</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ReadOperation</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;Read passenger information&quot;</span><span class="p">,</span> <span class="n">EndPoint</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;ns&quot;</span><span class="p">,</span> <span class="s">&quot;passengerList&quot;</span><span class="p">),</span>
                                   <span class="s">&quot;id&quot;</span><span class="p">,</span> <span class="s">&quot;firstName&quot;</span><span class="p">,</span> <span class="s">&quot;lastName&quot;</span><span class="p">,</span> <span class="s">&quot;address&quot;</span><span class="p">);</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">TransformOperation</span></code> to represent the concatenation of fields firstName and lastName:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">Operation</span> <span class="n">concat</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TransformOperation</span><span class="p">(</span><span class="s">&quot;Concat&quot;</span><span class="p">,</span> <span class="s">&quot;Concatenated fields&quot;</span><span class="p">,</span>
                                          <span class="n">Arrays</span><span class="p">.</span><span class="na">asList</span><span class="p">(</span><span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;firstName&quot;</span><span class="p">),</span>
                                          <span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;lastName&quot;</span><span class="p">)),</span> <span class="s">&quot;fullName&quot;</span><span class="p">);</span>
</pre></div>
</div>
<p>Another <code class="docutils literal notranslate"><span class="pre">TransformOperation</span></code> to represent the normalization of address:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">Operation</span> <span class="n">normalize</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TransformOperation</span><span class="p">(</span><span class="s">&quot;Normalize&quot;</span><span class="p">,</span> <span class="s">&quot;Normalized field&quot;</span><span class="p">,</span>
                                             <span class="n">Collections</span><span class="p">.</span><span class="na">singletonList</span><span class="p">(</span><span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;address&quot;</span><span class="p">)),</span>
                                             <span class="s">&quot;address&quot;</span><span class="p">);</span>
</pre></div>
</div>
<p>Finally <code class="docutils literal notranslate"><span class="pre">WriteOperation</span></code> to represent the fields are being written to the example CDAP dataset <code class="docutils literal notranslate"><span class="pre">passenger</span></code>::</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">Operation</span> <span class="n">write</span> <span class="o">=</span> <span class="k">new</span> <span class="n">WriteOperation</span><span class="p">(</span><span class="s">&quot;Write&quot;</span><span class="p">,</span> <span class="s">&quot;Wrote to passenger dataset&quot;</span><span class="p">,</span> <span class="n">EndPoint</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;ns&quot;</span><span class="p">,</span> <span class="s">&quot;passenger&quot;</span><span class="p">),</span>
                                     <span class="n">Arrays</span><span class="p">.</span><span class="na">asList</span><span class="p">(</span><span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;id&quot;</span><span class="p">),</span>
                                                   <span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Concat&quot;</span><span class="p">,</span> <span class="s">&quot;fullName&quot;</span><span class="p">),</span>
                                                   <span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Normalize&quot;</span><span class="p">,</span> <span class="s">&quot;address&quot;</span><span class="p">)));</span>
</pre></div>
</div>
<p>Note that both <code class="docutils literal notranslate"><span class="pre">TransformOperation</span></code> and <code class="docutils literal notranslate"><span class="pre">WriteOperation</span></code> take list of <code class="docutils literal notranslate"><span class="pre">InputField</span></code> as a parameter representing
input fields to the operation. <code class="docutils literal notranslate"><span class="pre">InputField</span></code> is a pair of origin and field name that identifies the input
unambiguously. For example, the <code class="docutils literal notranslate"><span class="pre">Write</span></code> operation above uses the address field that <code class="docutils literal notranslate"><span class="pre">Normalize</span></code> operation generates,
not the one that <code class="docutils literal notranslate"><span class="pre">Read</span></code> operation generates.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">initialize()</span></code> method, which is invoked at runtime before a MapReduce or Spark program is executed,
can submit the Field operations created above to the platform:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@Override</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">initialize</span><span class="p">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
  <span class="n">MapReduceContext</span> <span class="n">context</span> <span class="o">=</span> <span class="n">getContext</span><span class="p">();</span>
  <span class="n">List</span><span class="o">&lt;</span><span class="n">Operation</span><span class="o">&gt;</span> <span class="n">operations</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="p">();</span>

  <span class="n">Operation</span> <span class="n">read</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ReadOperation</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;Read passenger information&quot;</span><span class="p">,</span> <span class="n">EndPoint</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;ns&quot;</span><span class="p">,</span> <span class="s">&quot;passengerList&quot;</span><span class="p">),</span>
                                     <span class="s">&quot;id&quot;</span><span class="p">,</span> <span class="s">&quot;firstName&quot;</span><span class="p">,</span> <span class="s">&quot;lastName&quot;</span><span class="p">,</span> <span class="s">&quot;address&quot;</span><span class="p">);</span>
  <span class="n">operations</span><span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="n">read</span><span class="p">);</span>

  <span class="n">Operation</span> <span class="n">concat</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TransformOperation</span><span class="p">(</span><span class="s">&quot;Concat&quot;</span><span class="p">,</span> <span class="s">&quot;Concatenated fields&quot;</span><span class="p">,</span>
                                            <span class="n">Arrays</span><span class="p">.</span><span class="na">asList</span><span class="p">(</span><span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;firstName&quot;</span><span class="p">),</span>
                                            <span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;lastName&quot;</span><span class="p">)),</span> <span class="s">&quot;fullName&quot;</span><span class="p">);</span>
  <span class="n">operations</span><span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="n">concat</span><span class="p">);</span>

  <span class="n">Operation</span> <span class="n">normalize</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TransformOperation</span><span class="p">(</span><span class="s">&quot;Normalize&quot;</span><span class="p">,</span> <span class="s">&quot;Normalized field&quot;</span><span class="p">,</span>
                                               <span class="n">Collections</span><span class="p">.</span><span class="na">singletonList</span><span class="p">(</span><span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;address&quot;</span><span class="p">)),</span>
                                               <span class="s">&quot;address&quot;</span><span class="p">);</span>
  <span class="n">operations</span><span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="n">normalize</span><span class="p">);</span>

  <span class="n">Operation</span> <span class="n">write</span> <span class="o">=</span> <span class="k">new</span> <span class="n">WriteOperation</span><span class="p">(</span><span class="s">&quot;Write&quot;</span><span class="p">,</span> <span class="s">&quot;Wrote to passenger dataset&quot;</span><span class="p">,</span> <span class="n">EndPoint</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;ns&quot;</span><span class="p">,</span> <span class="s">&quot;passenger&quot;</span><span class="p">),</span>
                                       <span class="n">Arrays</span><span class="p">.</span><span class="na">asList</span><span class="p">(</span><span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;id&quot;</span><span class="p">),</span>
                                                     <span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Concat&quot;</span><span class="p">,</span> <span class="s">&quot;fullName&quot;</span><span class="p">),</span>
                                                     <span class="n">InputField</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="s">&quot;Normalize&quot;</span><span class="p">,</span> <span class="s">&quot;address&quot;</span><span class="p">)));</span>
  <span class="n">operations</span><span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="n">write</span><span class="p">);</span>

  <span class="c1">// Record field operation</span>
  <span class="n">context</span><span class="p">.</span><span class="na">record</span><span class="p">(</span><span class="n">operations</span><span class="p">);</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
<div class="section" id="field-lineage-for-cdap">
<h2>Field Lineage for CDAP 数据流管道<a class="headerlink" href="#field-lineage-for-cdap" title="Permalink to this headline">🔗</a></h2>
<p>Plugins in CDAP data pipelines can also record the field lineage. Currently, plugins of type <code class="docutils literal notranslate"><span class="pre">batchsource</span></code>,
<code class="docutils literal notranslate"><span class="pre">transform</span></code>, and <code class="docutils literal notranslate"><span class="pre">batchsink</span></code> are supported. The capability to record lineage is available in the <code class="docutils literal notranslate"><span class="pre">prepareRun()</span></code>
method of the plugin by using the context provided to the <code class="docutils literal notranslate"><span class="pre">prepareRun()</span></code> method.</p>
<p><strong>Note:</strong> Individual plugins can record multiple operations, and all operations recorded by single plugin must have unique names.</p>
<p>Consider the use case above of importing the passenger information but with a CDAP data pipeline that consists of
File Source, Concatenate, Address Normalizer, and Table Sink plugins.</p>
<p>These plugins can record the following field lineage operations:</p>
<p>File Source plugin can record the <code class="docutils literal notranslate"><span class="pre">FieldReadOperation</span></code>, which represents reading the passenger information stored
in CSV file to create the following fields id, firstName, lastName, and address. The fields belong to the output schema:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@Override</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepareRun</span><span class="p">(</span><span class="n">BatchSourceContext</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
  <span class="k">if</span> <span class="p">(</span><span class="n">config</span><span class="p">.</span><span class="na">getSchema</span><span class="p">()</span> <span class="o">!=</span> <span class="kc">null</span> <span class="o">&amp;&amp;</span> <span class="n">config</span><span class="p">.</span><span class="na">getSchema</span><span class="p">().</span><span class="na">getFields</span><span class="p">()</span> <span class="o">!=</span> <span class="kc">null</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">List</span><span class="o">&lt;</span><span class="n">Schema</span><span class="p">.</span><span class="na">Field</span><span class="o">&gt;</span> <span class="n">fields</span> <span class="o">=</span> <span class="n">config</span><span class="p">.</span><span class="na">getSchema</span><span class="p">().</span><span class="na">getFields</span><span class="p">();</span>
    <span class="c1">// Make sure the schema and fields are non null</span>
    <span class="n">FieldOperation</span> <span class="n">operation</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FieldReadOperation</span><span class="p">(</span><span class="s">&quot;Read&quot;</span><span class="p">,</span> <span class="s">&quot;Read from files&quot;</span><span class="p">,</span>
                                                      <span class="n">EndPoint</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="n">context</span><span class="p">.</span><span class="na">getNamespace</span><span class="p">(),</span> <span class="n">config</span><span class="p">.</span><span class="na">referenceName</span><span class="p">),</span>
                                                      <span class="n">fields</span><span class="p">.</span><span class="na">stream</span><span class="p">().</span><span class="na">map</span><span class="p">(</span><span class="n">Schema</span><span class="p">.</span><span class="na">Field</span><span class="p">::</span><span class="n">getName</span><span class="p">)</span>
                                                        <span class="p">.</span><span class="na">collect</span><span class="p">(</span><span class="n">Collectors</span><span class="p">.</span><span class="na">toList</span><span class="p">()));</span>
    <span class="n">context</span><span class="p">.</span><span class="na">record</span><span class="p">(</span><span class="n">Collections</span><span class="p">.</span><span class="na">singletonList</span><span class="p">(</span><span class="n">operation</span><span class="p">));</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Concatenate plugin concatenates the fields as represented by the plugin config and record the
<code class="docutils literal notranslate"><span class="pre">FieldTransformOperation</span></code> in its prepareRun method:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@Override</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepareRun</span><span class="p">(</span><span class="n">StageSubmitterContext</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
  <span class="n">FieldOperation</span> <span class="n">operation</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FieldTransformOperation</span><span class="p">(</span><span class="s">&quot;Concatenate&quot;</span><span class="p">,</span> <span class="s">&quot;Concatenated fields&quot;</span><span class="p">,</span>
                                                         <span class="n">Arrays</span><span class="p">.</span><span class="na">asList</span><span class="p">(</span><span class="n">config</span><span class="p">.</span><span class="na">fieldToConcatenate1</span><span class="p">,</span>
                                                                       <span class="n">config</span><span class="p">.</span><span class="na">fieldToConcatenate2</span><span class="p">),</span>
                                                         <span class="n">config</span><span class="p">.</span><span class="na">newFieldName</span><span class="p">);</span>
  <span class="n">context</span><span class="p">.</span><span class="na">record</span><span class="p">(</span><span class="n">Collections</span><span class="p">.</span><span class="na">singletonList</span><span class="p">(</span><span class="n">operation</span><span class="p">));</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Similarly Address Normalizer plugin can record the <code class="docutils literal notranslate"><span class="pre">FieldTransformOperation</span></code> representing address normalization:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@Override</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepareRun</span><span class="p">(</span><span class="n">StageSubmitterContext</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
  <span class="n">FieldOperation</span> <span class="n">operation</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FieldTransformOperation</span><span class="p">(</span><span class="s">&quot;Normalize&quot;</span><span class="p">,</span> <span class="s">&quot;Normalized field&quot;</span><span class="p">,</span>
                                                         <span class="n">Collections</span><span class="p">.</span><span class="na">singletonList</span><span class="p">(</span><span class="n">config</span><span class="p">.</span><span class="na">fieldToNormalize</span><span class="p">),</span>
                                                         <span class="n">config</span><span class="p">.</span><span class="na">fieldToConcatenate2</span><span class="p">);</span>
  <span class="n">context</span><span class="p">.</span><span class="na">record</span><span class="p">(</span><span class="n">Collections</span><span class="p">.</span><span class="na">singletonList</span><span class="p">(</span><span class="n">operation</span><span class="p">));</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Finally, Table sink can record <code class="docutils literal notranslate"><span class="pre">FieldWriteOperation</span></code> representing writing fields to dataset:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@Override</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepareRun</span><span class="p">(</span><span class="n">BatchSinkContext</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
  <span class="k">if</span> <span class="p">(</span><span class="n">schema</span><span class="p">.</span><span class="na">getFields</span><span class="p">()</span> <span class="o">!=</span> <span class="kc">null</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">FieldOperation</span> <span class="n">operation</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FieldWriteOperation</span><span class="p">(</span><span class="s">&quot;Write&quot;</span><span class="p">,</span> <span class="s">&quot;Wrote to CDAP Table&quot;</span><span class="p">,</span>
                                                       <span class="n">EndPoint</span><span class="p">.</span><span class="na">of</span><span class="p">(</span><span class="n">context</span><span class="p">.</span><span class="na">getNamespace</span><span class="p">(),</span> <span class="s">&quot;passenger&quot;</span><span class="p">),</span>
                                                       <span class="n">schema</span><span class="p">.</span><span class="na">getFields</span><span class="p">().</span><span class="na">stream</span><span class="p">().</span><span class="na">map</span><span class="p">(</span><span class="n">Schema</span><span class="p">.</span><span class="na">Field</span><span class="p">::</span><span class="n">getName</span><span class="p">)</span>
                                                         <span class="p">.</span><span class="na">collect</span><span class="p">(</span><span class="n">Collectors</span><span class="p">.</span><span class="na">toList</span><span class="p">()));</span>
    <span class="n">context</span><span class="p">.</span><span class="na">record</span><span class="p">(</span><span class="n">Collections</span><span class="p">.</span><span class="na">singletonList</span><span class="p">(</span><span class="n">operation</span><span class="p">));</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Discovery and Lineage" href="discovery-lineage.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Audit Logging" href="audit-logging.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>